{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# HopperBulletEnv with Twin Delayed DDPG (TD3)\n",
    "\n",
    "### Paper: https://arxiv.org/abs/1802.09477\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Start the Environment and Agent "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "current_dir=/home/rafaels/ml-agents/lib/python3.7/site-packages/pybullet_envs/bullet\n",
      "device:  cpu\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import pybullet_envs\n",
    "import pybullet_envs.bullet as bul\n",
    "\n",
    "#from torch.autograd import Variable\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import gym\n",
    "#import argparse\n",
    "import os\n",
    "\n",
    "import time\n",
    "from TwinDelayed_U import Actor, Critic, ReplayBuffer, TD3\n",
    "from collections import deque\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start_dim:  15 , action_dim:  3\n",
      "n max_action:  1.0 , threshold:  2500.0\n"
     ]
    }
   ],
   "source": [
    "start_timestep=1e4\n",
    "\n",
    "std_noise=0.03\n",
    "\n",
    "env = gym.make('HopperBulletEnv-v0')\n",
    "\n",
    "# Set seeds\n",
    "seed = 12345\n",
    "env.seed(seed)\n",
    "torch.manual_seed(seed)\n",
    "np.random.seed(seed)\n",
    "\n",
    "state = env.reset()\n",
    "state_dim = env.observation_space.shape[0]\n",
    "action_dim = env.action_space.shape[0] \n",
    "max_action = float(env.action_space.high[0])\n",
    "threshold = env.spec.reward_threshold\n",
    "\n",
    "print('start_dim: ', state_dim, ', action_dim: ', action_dim)\n",
    "print('n max_action: ', max_action, ', threshold: ', threshold)\n",
    "\n",
    "agent = TD3(state_dim, action_dim, max_action)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Traning the agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save(agent = agent, filename='checkpnt, directory = 'dir_chkpoint')     \n",
    "def save(agent, filename, directory):\n",
    "    torch.save(agent.actor.state_dict(), '%s/%s_actor.pth' % (directory, filename))\n",
    "    torch.save(agent.critic.state_dict(), '%s/%s_critic.pth' % (directory, filename))\n",
    "    torch.save(agent.actor_target.state_dict(), '%s/%s_actor_t.pth' % (directory, filename))\n",
    "    torch.save(agent.critic_target.state_dict(), '%s/%s_critic_t.pth' % (directory, filename))   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Low in action space:  [-1. -1. -1.] , High:  [1. 1. 1.] , Action_dim:  3\n",
      "Ep. 40, Timestep 488,  Ep.Timesteps 9, Score: 18.06, Avg.Score: 19.97, Max.Score: 26.26,  Time: 00:00:07 \n",
      "Ep. 80, Timestep 963,  Ep.Timesteps 14, Score: 22.76, Avg.Score: 20.10, Max.Score: 33.08,  Time: 00:00:17 \n",
      "Ep. 120, Timestep 1442,  Ep.Timesteps 11, Score: 20.15, Avg.Score: 20.36, Max.Score: 33.08,  Time: 00:00:28 \n",
      "Ep. 160, Timestep 1940,  Ep.Timesteps 7, Score: 20.11, Avg.Score: 20.18, Max.Score: 33.08,  Time: 00:00:40 \n",
      "Ep. 200, Timestep 2436,  Ep.Timesteps 14, Score: 21.62, Avg.Score: 20.37, Max.Score: 30.01,  Time: 00:00:52 \n",
      "Ep. 240, Timestep 2997,  Ep.Timesteps 12, Score: 19.13, Avg.Score: 20.60, Max.Score: 38.31,  Time: 00:01:05 \n",
      "Ep. 280, Timestep 3488,  Ep.Timesteps 7, Score: 16.72, Avg.Score: 20.58, Max.Score: 43.57,  Time: 00:01:17 \n",
      "Ep. 320, Timestep 4001,  Ep.Timesteps 11, Score: 15.33, Avg.Score: 20.27, Max.Score: 43.57,  Time: 00:01:29 \n",
      "Ep. 360, Timestep 4469,  Ep.Timesteps 8, Score: 19.94, Avg.Score: 20.11, Max.Score: 43.57,  Time: 00:01:41 \n",
      "Ep. 400, Timestep 5022,  Ep.Timesteps 8, Score: 20.08, Avg.Score: 20.41, Max.Score: 42.06,  Time: 00:01:54 \n",
      "Ep. 440, Timestep 5491,  Ep.Timesteps 9, Score: 18.44, Avg.Score: 20.27, Max.Score: 42.06,  Time: 00:02:06 \n",
      "Ep. 480, Timestep 5997,  Ep.Timesteps 13, Score: 16.19, Avg.Score: 20.62, Max.Score: 28.73,  Time: 00:02:18 \n",
      "Ep. 520, Timestep 6459,  Ep.Timesteps 10, Score: 20.84, Avg.Score: 20.48, Max.Score: 28.73,  Time: 00:02:29 \n",
      "Ep. 560, Timestep 6980,  Ep.Timesteps 17, Score: 23.77, Avg.Score: 20.50, Max.Score: 32.13,  Time: 00:02:41 \n",
      "Ep. 600, Timestep 7442,  Ep.Timesteps 10, Score: 17.04, Avg.Score: 20.25, Max.Score: 32.13,  Time: 00:02:53 \n",
      "Ep. 640, Timestep 7905,  Ep.Timesteps 9, Score: 18.91, Avg.Score: 19.72, Max.Score: 29.94,  Time: 00:03:04 \n",
      "Ep. 680, Timestep 8376,  Ep.Timesteps 7, Score: 16.94, Avg.Score: 19.41, Max.Score: 32.95,  Time: 00:03:15 \n",
      "Ep. 720, Timestep 8865,  Ep.Timesteps 25, Score: 27.80, Avg.Score: 19.72, Max.Score: 32.95,  Time: 00:03:27 \n",
      "Ep. 760, Timestep 9327,  Ep.Timesteps 9, Score: 16.96, Avg.Score: 19.90, Max.Score: 30.39,  Time: 00:03:38 \n",
      "Ep. 800, Timestep 9843,  Ep.Timesteps 7, Score: 17.46, Avg.Score: 19.98, Max.Score: 31.72,  Time: 00:03:51 \n",
      "Ep. 840, Timestep 10263,  Ep.Timesteps 9, Score: 13.12, Avg.Score: 18.12, Max.Score: 31.74,  Time: 00:04:01 \n",
      "Ep. 880, Timestep 10653,  Ep.Timesteps 10, Score: 14.02, Avg.Score: 15.42, Max.Score: 31.74,  Time: 00:04:11 \n",
      "Ep. 920, Timestep 11085,  Ep.Timesteps 29, Score: -1.35, Avg.Score: 13.27, Max.Score: 18.58,  Time: 00:04:21 \n",
      "Ep. 960, Timestep 15505,  Ep.Timesteps 43, Score: 69.46, Avg.Score: 45.38, Max.Score: 916.17,  Time: 00:06:08 \n",
      "Ep. 1000, Timestep 17469,  Ep.Timesteps 52, Score: 84.49, Avg.Score: 70.66, Max.Score: 916.17,  Time: 00:06:57 \n",
      "Ep. 1040, Timestep 19262,  Ep.Timesteps 48, Score: 79.20, Avg.Score: 70.21, Max.Score: 125.44,  Time: 00:07:42 \n",
      "Ep. 1080, Timestep 21279,  Ep.Timesteps 65, Score: 108.16, Avg.Score: 79.63, Max.Score: 125.44,  Time: 00:08:31 \n",
      "Ep. 1120, Timestep 24147,  Ep.Timesteps 78, Score: 121.71, Avg.Score: 93.50, Max.Score: 138.22,  Time: 00:09:42 \n",
      "Ep. 1160, Timestep 28026,  Ep.Timesteps 145, Score: 163.03, Avg.Score: 116.60, Max.Score: 884.07,  Time: 00:11:17 \n",
      "Ep. 1200, Timestep 43589,  Ep.Timesteps 71, Score: 115.74, Avg.Score: 204.15, Max.Score: 884.07,  Time: 00:17:48 \n",
      "Ep. 1240, Timestep 69241,  Ep.Timesteps 1000, Score: 914.77, Avg.Score: 369.27, Max.Score: 951.01,  Time: 00:29:03 \n",
      "Ep. 1280, Timestep 88365,  Ep.Timesteps 1000, Score: 893.08, Avg.Score: 445.49, Max.Score: 951.01,  Time: 00:40:13 \n",
      "Ep. 1320, Timestep 124223,  Ep.Timesteps 95, Score: 140.08, Avg.Score: 617.19, Max.Score: 1095.19,  Time: 01:03:16 \n",
      "Ep. 1360, Timestep 150962,  Ep.Timesteps 1000, Score: 1640.19, Avg.Score: 734.71, Max.Score: 1699.38,  Time: 01:20:47 \n",
      "Ep. 1400, Timestep 187630,  Ep.Timesteps 1000, Score: 1815.40, Avg.Score: 1099.67, Max.Score: 2036.40,  Time: 01:47:03 \n",
      "Ep. 1440, Timestep 224909,  Ep.Timesteps 1000, Score: 1952.30, Avg.Score: 1548.38, Max.Score: 2071.15,  Time: 02:14:20 \n",
      "Ep. 1480, Timestep 262641,  Ep.Timesteps 1000, Score: 1896.34, Avg.Score: 1786.84, Max.Score: 2151.99,  Time: 02:42:16 \n",
      "Ep. 1520, Timestep 299845,  Ep.Timesteps 1000, Score: 2019.45, Avg.Score: 1843.53, Max.Score: 2249.05,  Time: 03:10:09 \n",
      "Ep. 1560, Timestep 339109,  Ep.Timesteps 1000, Score: 1931.54, Avg.Score: 1956.42, Max.Score: 2295.53,  Time: 03:39:46 \n",
      "Ep. 1600, Timestep 377547,  Ep.Timesteps 1000, Score: 2122.79, Avg.Score: 1995.30, Max.Score: 2295.53,  Time: 04:09:03 \n",
      "Ep. 1640, Timestep 413346,  Ep.Timesteps 1000, Score: 2342.34, Avg.Score: 2019.53, Max.Score: 2342.34,  Time: 04:36:31 \n",
      "Ep. 1680, Timestep 450201,  Ep.Timesteps 1000, Score: 2399.96, Avg.Score: 2039.13, Max.Score: 2447.73,  Time: 05:05:00 \n",
      "Ep. 1720, Timestep 485725,  Ep.Timesteps 1000, Score: 2343.44, Avg.Score: 2056.50, Max.Score: 2447.73,  Time: 05:32:37 \n",
      "Ep. 1760, Timestep 525725,  Ep.Timesteps 1000, Score: 2368.79, Avg.Score: 2158.43, Max.Score: 2505.90,  Time: 06:04:01 \n",
      "Ep. 1800, Timestep 561978,  Ep.Timesteps 1000, Score: 2455.13, Avg.Score: 2209.27, Max.Score: 2505.90,  Time: 06:32:36 \n",
      "Ep. 1840, Timestep 599396,  Ep.Timesteps 1000, Score: 2396.38, Avg.Score: 2211.40, Max.Score: 2488.39,  Time: 07:02:20 \n",
      "Ep. 1880, Timestep 636981,  Ep.Timesteps 1000, Score: 2433.04, Avg.Score: 2180.01, Max.Score: 2479.78,  Time: 07:32:15 \n",
      "Ep. 1920, Timestep 676955,  Ep.Timesteps 1000, Score: 2431.96, Avg.Score: 2315.40, Max.Score: 2534.61,  Time: 08:04:08 \n",
      "Ep. 1960, Timestep 715945,  Ep.Timesteps 1000, Score: 2478.93, Avg.Score: 2408.96, Max.Score: 2566.69,  Time: 08:35:30 \n",
      "Ep. 2000, Timestep 753961,  Ep.Timesteps 1000, Score: 2547.92, Avg.Score: 2405.01, Max.Score: 2614.92,  Time: 09:06:12 \n",
      "Ep. 2040, Timestep 788674,  Ep.Timesteps 1000, Score: 2498.11, Avg.Score: 2288.78, Max.Score: 2614.92,  Time: 09:34:16 \n",
      "Ep. 2080, Timestep 826164,  Ep.Timesteps 1000, Score: 2219.40, Avg.Score: 2261.40, Max.Score: 2614.68,  Time: 10:04:47 \n",
      "Ep. 2120, Timestep 866164,  Ep.Timesteps 1000, Score: 2520.67, Avg.Score: 2378.04, Max.Score: 2579.46,  Time: 10:37:25 \n",
      "Ep. 2160, Timestep 904083,  Ep.Timesteps 1000, Score: 2500.44, Avg.Score: 2381.53, Max.Score: 2579.96,  Time: 11:08:30 \n",
      "Ep. 2200, Timestep 938654,  Ep.Timesteps 1000, Score: 2528.00, Avg.Score: 2301.14, Max.Score: 2579.96,  Time: 11:36:58 \n",
      "Ep. 2240, Timestep 971015,  Ep.Timesteps 1000, Score: 2561.32, Avg.Score: 2147.88, Max.Score: 2579.96,  Time: 12:03:37 \n",
      "Ep. 2280, Timestep 997428,  Ep.Timesteps 340, Score: 844.84, Avg.Score: 1909.89, Max.Score: 2591.64,  Time: 12:25:56 \n",
      "Ep. 2320, Timestep 1033442,  Ep.Timesteps 1000, Score: 2563.26, Avg.Score: 1972.70, Max.Score: 2617.38,  Time: 12:55:26 \n",
      "Ep. 2360, Timestep 1071335,  Ep.Timesteps 84, Score: 167.19, Avg.Score: 2275.20, Max.Score: 2627.72,  Time: 13:28:16 \n",
      "Ep. 2400, Timestep 1104155,  Ep.Timesteps 1000, Score: 2583.74, Avg.Score: 2281.79, Max.Score: 2627.72,  Time: 13:55:50 \n",
      "Ep. 2440, Timestep 1140725,  Ep.Timesteps 1000, Score: 2527.41, Avg.Score: 2255.09, Max.Score: 2715.20,  Time: 14:27:49 \n",
      "Ep. 2480, Timestep 1175807,  Ep.Timesteps 1000, Score: 2625.29, Avg.Score: 2351.07, Max.Score: 2715.20,  Time: 14:58:54 \n",
      "Ep. 2520, Timestep 1215253,  Ep.Timesteps 1000, Score: 2608.98, Avg.Score: 2378.45, Max.Score: 2727.90,  Time: 15:34:16 \n",
      "Ep. 2560, Timestep 1249969,  Ep.Timesteps 1000, Score: 2607.16, Avg.Score: 2373.62, Max.Score: 2727.90,  Time: 16:05:38 \n",
      "Ep. 2600, Timestep 1288271,  Ep.Timesteps 1000, Score: 2557.92, Avg.Score: 2381.32, Max.Score: 2707.86,  Time: 16:40:22 \n",
      "Ep. 2640, Timestep 1323095,  Ep.Timesteps 260, Score: 635.40, Avg.Score: 2242.48, Max.Score: 2709.74,  Time: 17:12:44 \n",
      "Ep. 2680, Timestep 1352728,  Ep.Timesteps 1000, Score: 2620.84, Avg.Score: 2020.79, Max.Score: 2709.74,  Time: 17:38:54 \n",
      "Ep. 2720, Timestep 1388128,  Ep.Timesteps 1000, Score: 2472.57, Avg.Score: 1941.28, Max.Score: 2665.82,  Time: 18:10:49 \n",
      "Ep. 2760, Timestep 1427140,  Ep.Timesteps 1000, Score: 2321.51, Avg.Score: 2337.54, Max.Score: 2684.57,  Time: 18:46:20 \n",
      "Ep. 2800, Timestep 1464271,  Ep.Timesteps 1000, Score: 2560.58, Avg.Score: 2424.37, Max.Score: 2684.57,  Time: 19:20:11 \n",
      "Ep. 2840, Timestep 1500370,  Ep.Timesteps 1000, Score: 2583.03, Avg.Score: 2349.95, Max.Score: 2680.23,  Time: 19:53:12 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep. 2880, Timestep 1537112,  Ep.Timesteps 1000, Score: 2645.05, Avg.Score: 2318.04, Max.Score: 2680.23,  Time: 20:26:58 \n",
      "Ep. 2920, Timestep 1574451,  Ep.Timesteps 1000, Score: 2480.05, Avg.Score: 2351.63, Max.Score: 2699.69,  Time: 21:01:22 \n",
      "Ep. 2960, Timestep 1607917,  Ep.Timesteps 1000, Score: 2564.41, Avg.Score: 2273.12, Max.Score: 2699.69,  Time: 21:32:13 \n",
      "Ep. 3000, Timestep 1642912,  Ep.Timesteps 157, Score: 378.34, Avg.Score: 2199.56, Max.Score: 2724.86,  Time: 22:05:13 \n",
      "Ep. 3040, Timestep 1677333,  Ep.Timesteps 1000, Score: 2545.76, Avg.Score: 2116.48, Max.Score: 2724.86,  Time: 22:36:08 \n",
      "Ep. 3080, Timestep 1715076,  Ep.Timesteps 1000, Score: 2542.09, Avg.Score: 2165.23, Max.Score: 2693.50,  Time: 23:10:47 \n",
      "Ep. 3120, Timestep 1753646,  Ep.Timesteps 1000, Score: 2520.82, Avg.Score: 2333.67, Max.Score: 2657.68,  Time: 23:46:14 \n",
      "Ep. 3160, Timestep 1786406,  Ep.Timesteps 1000, Score: 2566.96, Avg.Score: 2261.97, Max.Score: 2661.73,  Time: 24:16:20 \n",
      "Ep. 3200, Timestep 1826406,  Ep.Timesteps 1000, Score: 2633.13, Avg.Score: 2338.62, Max.Score: 2689.78,  Time: 24:53:04 \n",
      "Ep. 3240, Timestep 1864420,  Ep.Timesteps 1000, Score: 2590.85, Avg.Score: 2506.69, Max.Score: 2714.34,  Time: 25:28:08 \n",
      "Environment solved with Average Score:  2506.6920692399012\n"
     ]
    }
   ],
   "source": [
    "# Twin Delayed Deep Deterministic (TD3) policy gradient algorithm\n",
    "def twin_ddd_train(n_episodes=15000, save_every=10, print_env=40):\n",
    "\n",
    "    scores_deque = deque(maxlen=100)\n",
    "    scores_array = []\n",
    "    avg_scores_array = []    \n",
    "\n",
    "    time_start = time.time()                    # Init start time\n",
    "    replay_buf = ReplayBuffer()                 # Init ReplayBuffer\n",
    "    \n",
    "    timestep_after_last_save = 0\n",
    "    total_timesteps = 0\n",
    "    \n",
    "    low = env.action_space.low\n",
    "    high = env.action_space.high\n",
    "    \n",
    "    print('Low in action space: ', low, ', High: ', high, ', Action_dim: ', action_dim)\n",
    "            \n",
    "    for i_episode in range(1, n_episodes+1):\n",
    "        \n",
    "        timestep = 0\n",
    "        total_reward = 0\n",
    "        \n",
    "        # Reset environment\n",
    "        state = env.reset()\n",
    "        done = False\n",
    "        \n",
    "        while True:\n",
    "            \n",
    "            # Select action randomly or according to policy\n",
    "            if total_timesteps < start_timestep:\n",
    "                action = env.action_space.sample()\n",
    "            else:\n",
    "                action = agent.select_action(np.array(state))\n",
    "                if std_noise != 0: \n",
    "                    shift_action = np.random.normal(0, std_noise, size=action_dim)\n",
    "                    action = (action + shift_action).clip(low, high)\n",
    "            \n",
    "            # Perform action\n",
    "            new_state, reward, done, _ = env.step(action) \n",
    "            done_bool = 0 if timestep + 1 == env._max_episode_steps else float(done)\n",
    "            total_reward += reward                          # full episode reward\n",
    "\n",
    "            # Store every timestep in replay buffer\n",
    "            replay_buf.add((state, new_state, action, reward, done_bool))\n",
    "            state = new_state\n",
    "\n",
    "            timestep += 1     \n",
    "            total_timesteps += 1\n",
    "            timestep_after_last_save += 1\n",
    "\n",
    "            if done:                                       # done ?\n",
    "                break                                      # save score\n",
    "\n",
    "        scores_deque.append(total_reward)\n",
    "        scores_array.append(total_reward)\n",
    "\n",
    "        avg_score = np.mean(scores_deque)\n",
    "        avg_scores_array.append(avg_score)\n",
    "        \n",
    "        max_score = np.max(scores_deque)\n",
    "        \n",
    "        # train_by_episode(time_start, i_episode) \n",
    "        s = (int)(time.time() - time_start)\n",
    "        if i_episode % print_env == 0 or (len(scores_deque) == 100 and avg_score > threshold):\n",
    "            print('Ep. {}, Timestep {},  Ep.Timesteps {}, Score: {:.2f}, Avg.Score: {:.2f}, Max.Score: {:.2f},  Time: {:02}:{:02}:{:02} '\\\n",
    "                .format(i_episode, total_timesteps, timestep, \\\n",
    "                        total_reward, avg_score, max_score, s//3600, s%3600//60, s%60))     \n",
    "\n",
    "        agent.train(replay_buf, timestep)\n",
    "\n",
    "        # Save episode if more than save_every=5000 timesteps\n",
    "        if timestep_after_last_save >= save_every and i_episode > 0:\n",
    "\n",
    "            timestep_after_last_save %= save_every            \n",
    "            save(agent, 'chpnt_1', 'dir_chk_003')  \n",
    "        \n",
    "        if len(scores_deque) == 100 and avg_score >= threshold:\n",
    "            print('Environment solved with Average Score: ',  avg_score )\n",
    "            break \n",
    "\n",
    "    return scores_array, avg_scores_array\n",
    "\n",
    "scores, avg_scores = twin_ddd_train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "save(agent, 'chpnt_2', 'dir_chk_003')  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length of scores:  3240 , len of avg_scores:  3240\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi0AAAEGCAYAAACpaqUaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOydd3iUVfbHP3cmk0ZIICQEEkpoEUITBAuCgoqAotgVu+sqa1lXXeu66q5lZVfRXXt37YqIP11BEawgFkB670oPJCQESJmZ+/tjZpJJMn3eKZmcz/Pkycx973vf807K/b7nnnOu0lojCIIgCIIQ75hibYAgCIIgCEIgiGgRBEEQBKFZIKJFEARBEIRmgYgWQRAEQRCaBSJaBEEQBEFoFiTF2oBIkJOTowsLC2NthiAIQrNi0aJFe7XWubG2QxC8kZCipbCwkIULF8baDEEQhGaFUmprrG0QBF/I8pAgCIIgCM0CES2CIAiCIDQLRLQIgiAIgtAsSMiYFkEQBCF+WLRoUfukpKSXgX7Iw7LgHTuwwmq1/v6oo47a46mDiBZBEAQhoiQlJb3coUOHPrm5uWUmk0k2vBM8YrfbVUlJSfGuXbteBs701EcUryAIghBp+uXm5laIYBF8YTKZdG5ubjkOj5znPlG0RxAEQWiZmESwCIHg/D3xqk1EtAiC0Ow4XGPjw0Xb0Lp5zYPrdx+ItQmC0KwR0SIIQtzyv6U7PE70D85YxZ8/WMqPm0p9nl9jtXOgqjZS5gXF5yt2MfqJ75ixbKfhY3+7roT5G/ditdkNHzuRuPPOOzv07Nmzb1FRUXHv3r2Lv/rqq1axtkkIDgnEFQQhbvnju4sB2DL59Abtu8urADhYba1rO/e5+SzaWsbmR05DKQXAFa/+zA+b9jU5P1Ks3lmBSSmO6NC6ybG1uw44v1dw+oCOhl1Ta80Vr/4MwEVDOzP53AGGjZ1IzJkzp9WsWbPaLF++fFVaWpreuXNnUnV1tQp1vNraWiwWi5EmCgEgnhZBEJodrkWhA9X1XpRFW8sAGnhffti0L6hxF20to9pqC9mucf+Zy5h/f+e3356KKgrvmsGcVbtDus7CLaU8MXsdANN/2V7XPnO58V6cRGH79u2W7Oxsa1pamgbo2LGjtbCwsPbbb79NHzRoUO8jjjiiuH///n3KyspMhw4dUuedd15hUVFRcZ8+fYr/97//tQZ48skn25100kk9jz322KJhw4YdAXDvvffm9evXr09RUVHxLbfckh/Le2wJiKdFEIRmxcFqKzVWxzLILe8vZVDnthTm1Hv5q0IUHRv2VHLuc/O5/LiuPDDBa/ICy7bt58ynv+ej64cxqEtbAE55/Ft6e/CuAPy0aR+rdlag0W5jlAPwzs+/ckpxXlB27q2s5rznfwBgz4FqftpcL8yaQ4TP7dOWdl6360C6kWMWdWh96NHzBv7mq89ZZ51V8cgjj+QXFhb2Gz58eMXEiRNLTz755IOXXHJJj7fffnvjiSeeeKi0tNSUkZFhf+ihh/KUUqxbt27V4sWLU0877bReGzduXAGwcuXK9GXLlq3My8uzTZ8+PXPDhg2py5YtW6215pRTTun52WefZYwbN67SyPsT6hFPiyAIcUWN1c65z83nv99v9ni87/2zmLdhb937kY990+C4zRbc1D1r5S4K75rBOmfszMzlu5i3fq/X/l+vKQHgqzX1ta827KnkUy+xKhe++CN//9+quvdbSw9xqNYhrEzK++rE4RpbAzte/G4jC7aUMuShOXVt7/78K5tKDta9b2ZxyVElKyvLvmLFilVPP/301tzcXOsVV1zRY8qUKbnt27evPfHEEw8BZGdn2y0WC/Pnz8+47LLL9gEMGjSoKj8/v2b58uWpACNGjKjIy8uzAXz++eeZ3333XWZxcXFx3759izdu3Ji6Zs2a1NjdZeIjnhZBEOKKLfsOsmhrWd1yT7BY7XasNjuT3lzks9/8DXs5pns7XpnrEEeumJO9ldVc+spPXuNgzM5HPbsfhfDuz79yYlFu3ft/z1kPwMdLdvDxkh0NxnJn6sLfmLViFxazic9X7mJQlzZ8dP3x/GPmGp/XC8SmeMCfRySSJCUlMX78+APjx48/MGDAgMPPP/98rv+zGpKenl4X7ay15uabb955++23e1e5gqGIp0UQhJhSUVVL2cGauvdJpuBjI1+dV++Vsdo18zfu40s3T8iMZTu59f0l2OyOSf27dSVc/PJPPP/tRmrtjjnIPagXYOqChnNr4V0z+NsnK+tiSGqsdhZt9Zy9VH64lrunL+fSl3/yabfZw73eMW0ZX67Zw+crdwGw+Nf97K2s9jmOi0M1tib3IThYunRpyvLly1Nc7xcvXpzWq1evqj179li+/fbbdICysjJTbW0txx9/fOVbb72VDbBs2bKUnTt3Jg8YMKCq8Zjjxo2rePPNN3PKy8tNAJs3b7Zs375dnAERRD5cQWhhHKqxMnf9Xk4syiXVYo7J9Wet3MVZRxaglGLIQ3PqYlRuHNWTbjlNs1AL75rBUxMH0bN9hscxH/i0fvmlxmrncmc2jYsb3vkFgCuGFTKwcxt2ObOPHp21ti4W5eV5DZej7vhwGRcM7QzAb6WHAPjv/C11x1+au5mX5m6mVXLTz/DUJ74FYNPeg02OufOrc1x/uC8J+WP4P79i8X2nBty/pVBRUWG+6aabulRUVJjNZrMuLCysfv3117euW7du70033dSlqqrKlJqaav/uu+/W3XHHHXsuv/zyrkVFRcVms5kXXnhhiyuA151zzjmnYuXKlalDhw7tDQ4vzNtvv725oKBAlGOEUM2tOFMgDBkyRC9cuDDWZghCXDLxxR/5YdM+TumTx8tXDInKNVfuKCcz1cKWfQe57BWHoJg66TgO1Vi58rUFUbEB4M2rj2ZEr1zeX/Ard3643G//LZNPR2vNkQ/MpvxwZOq9LL53NG1bJXPbB0uZtmibIWOGmuKtlFqktTb8l2Lp0qVbBg4cKEsoQkAsXbo0Z+DAgYWejomnRRBaELsrqurSgOesDi3dNlhqbXZOf3Ie0HDp5/0Fv/HhL8ZM0oFSXevw6AT6rPbz5lIyUpIiJlgABj04mw0PjzNMsAhCIiMxLYLQgrj3/1Y0eH+4xsYr8zZzwfM/8Muv9YGvVbU2n5VkX567iTd/2BLQNd0nY6u9Xi1EW7AAVDuXoQL1L1/wwg+UBBhPEg6//Lo/4tcQhERAPC2C0EK4deoSvmhUzOzc5+azamcFAOc8O79uWeHkKd+yff/hBssMh2tsvPnjFq4e3p2HZqwGHEGiD57Vj1YpDf+VbNhTySmPfxvJ2wmJtbsPcFJNe+77eIX/zk6uaBQfEwkueOEHw8bq0zHTsLEEId4Q0SIICUxltZWygzV0zk5vUDnVhUuwNGb7/sNN2s54eh4b9lTSMSutrm364u1MX7ydVQ+MIT3Z8e9k/sa9XPyS76yZWPHkl+tZ8tt+aoOs5dKcuHBIp1ibIAgRQ5aHBCHB0FozdcFv1FjtjH9yLiP+9XVd4bRAcF8mcvHR4m1s2OMo8unaD8idu6fXB7Xe/sGyEKyOHt+tK4m1CRHl/YUSGyMkLhETLUqpzkqpr5VSq5RSK5VSf3K2/00ptV0ptcT5dZrbOXcrpTYopdYqpca4tY91tm1QSt0VKZsFIRH4dNlO7vhwGU9/tZ4t+xwptac+4X8/HHAEzZ7z7Py691+v2UPhXTO45f2lPs/7eMkOCu+aQVWtzWPtESF6yE7PQiITSU+LFfiz1roYOBa4QSlV7Dz2hNb6SOfXTADnsYuAvsBY4FmllFkpZQaeAcYBxcBEt3EEISGprLb6LCXvi7JDjkJtT361Iehze93zWYP3V/03uHTkT5buCLj2iGA8E0zzGFMzO9ZmxC1vvvlmG6XUUYsXL46LUvu7du0yH3PMMUXp6emDLr/88i7ux+bOnZteVFRU3KVLl35XXnllZ7uzCOLu3bvNw4YN69W1a9d+w4YN61VSUmJIsaUTTzyx5969e8Ma69NPP209atSonkbY442IiRat9U6t9S/O1weA1UCBj1MmAO9prau11puBDcDRzq8NWutNWusa4D1nX0FIWG59fwmXvvITO/Yf9pnF4wlXobZYcMe0+F4aSmRuT3qP/yQ/y7haES3eeO+997IHDx5c+cYbb2TH2haA9PR0/cADD+z429/+1mRN7/rrr+/63HPPbd2yZcuKTZs2pU6bNi0T4P777+84cuTIA1u3bl0xcuTIA/fdd18HI2z59ttvN+Tk5IS+xXmUiEpMi1KqEBgEuKLzblRKLVNKvaqUautsKwDc62Zvc7Z5a298jWuVUguVUgtLShJ7zVpIfNY740dmrdxF/799wWfLd/LtuhJ27D/MDC8b84EjVdmV2SO0LG5I+gSA3rZ1MbYkPikvLzctWLAg47XXXtvy0Ucf1YmW8ePHd3/vvfeyXO/PPffcwtdee63tgQMHTKeddlr3Hj169B09enSPAQMG9P7uu++a7E798ccft+7Tp09xUVFR8fnnn194+PBhBVBQUND/lltuyS8uLu5TVFRU7Mm7k5mZaR8zZkxlampqgyeNrVu3WiorK00nn3zyQZPJxCWXXLLv//7v/9oCfP75520mTZq0D2DSpEn7Pvvss7aNx7VarUyaNKlTv379+hQVFRU/+uijOeDwhAwZMuSIkSNH9iwsLOx38cUXd7HZHDqloKCg/86dO5MqKipMI0eO7HnEEUcU9+rVq+9LL73U1td9Tps2LbNbt259i4uL+0ybNq2Ny4aKigrT+eefX9i/f/8+ffr0KX7rrbfaACxcuDC1f//+fXr37l1cVFRU7L61QiBEPHtIKZUBfAjcrLWuUEo9BzyIo1TCg8AU4HfhXkdr/SLwIjgq4oY7niDEihqrnc3O8u8/b3bsbXPd27806HP6gIYVTyurrZQfrq0LlhWiT6rFRFVtbLxcivrrvptyPpfGxIoA+b8bOrNnVZPJPyzaFx/irGd8bsT4zjvvtBk5cmT5gAEDqtu2bWudO3du+ogRIw5dcMEFpVOnTm170UUXlVdVVanvv/8+8/XXX9/6r3/9q32bNm1sGzduXLlgwYLU4447rm/jMQ8dOqQmTZrU7Ysvvlg7YMCA6rPPPrvw0Ucfzb3vvvv2AOTk5FhXrVq1evLkybmTJ0/Oe//997cGcjtbt261dOzYsc7F2rVr15qdO3daAPbt25fUtWvXWoDOnTvX7tu3r8k8/u9//zsnKyvLtmLFitWHDx9WQ4cO7X3GGWdUACxfvrzV4sWLVxQVFdWccMIJvd544422V111VV30/fTp0zM7dOhQ+80332xwXs/s7T5vu+22khtvvLFw9uzZa/v27Vs9fvz47q5x/vKXv3QcNWpUxQcffLBl79695iFDhvQ588wzK5566qnc66+/fvd1111XWlVVpazW4HY8iKinRSllwSFY3tZaTwfQWu/WWtu01nbgJRzLPwDbgc5up3dytnlrF4RmTfnhWg7XNPXGujbKA7xulLdyR3mD9+c//wPHT/4qKjVF4oGbT+kVUL/zjope+u+bVx8TtWuN7dtwRaA1jhT1KbXn8UbqJVGzozkxderU7IkTJ5YBnHvuuaVvvvlmNsB5551X/sMPP7Q+fPiwmjZtWtbRRx99ICMjQ8+fPz9j4sSJpQBDhw6tKioqahKstXTp0tROnTpVDxgwoBrgyiuv3Ddv3rzWruMXX3xxGcDRRx996LfffgvKoxAIJpMJpZoGvs+ZMydz6tSp7Xr37l08aNCgPmVlZUmrVq1KBejfv//B4uLimqSkJC644ILSuXPnNtjQa/DgwYfnzp2bed111xV8/vnnGe3atbN5u88lS5akdurUqbp///7VLo+Qa5xvvvkm84knnujYu3fv4uHDhx9RXV2tNmzYkHzccccdnDJlSsd77rmnw/r165MzMjKCcjJEzNOiHJ/kK8BqrfXjbu0dtdYu//bZgKvK0yfAO0qpx4F8oBfwM6CAXkqpbjjEykXAxZGyWxCixcC/f0FBmzS+v+skr30WbGmafgxw+pPzeOlyxxYxo4vzWO2l3kqiYnL7R51FJU9YnqWX2s5pNY9wgHRM2NE4Pptolcfvl5/lv5NBjOvfoYG4zVQOz9xO2qGI8+wtPx6RSLB7927zjz/+2Hrt2rVpN954IzabTSmltN1u35aenq6PPfbYA9OnT898//3321500UWet+4OgdTUVA2QlJSkrVZrwD+Yrl271ro8KwBbt25Ndnle2rVrZ926daula9eutVu3brVkZ2c3cVVordWUKVN+Pffccxv8Y/j0009bNxY5jd8PGDCg+pdffln14YcfZt17770Fc+bMqTj33HODLtmstWbatGkbBg4c2ODJa/DgwVUjRow4+NFHH2WNHz++11NPPbX1zDPPDLgmQyQ9LccDlwEnNUpv/pdSarlSahkwCrgFQGu9EpgKrAI+B25wemSswI3ALBzBvFOdfQWh2eOpiFugGcPXvLGQa95YyHVvLQr6ur287JbcXHD/iE4x/cJJ5iV0NpXw+6SZgGZuq7v4OvnPKODda44N+3om7Fxj/pTOKjr7NQVLFg7RUq6b7pAtwJtvvtn27LPPLt2xY8fy7du3L9+1a9eyTp061cyaNSsD4MILLyz773//m7NgwYLWron+uOOOq3zvvffaAixatCh13bp1aY3HHThwYNX27duTV6xYkQLwxhtvtBsxYkTgRZG80LVr19qMjAz7l19+2cput/P222+3mzBhwn6AMWPG7H/hhRfaAbzwwgvtxo4d20RQjB49uvy5557Lra6uVgDLli1LqaioMIFjeWjNmjXJNpuNadOmZTe2d8uWLZbWrVvbr7/++tJbb71115IlS9K93eeRRx5ZtX379uSVK1emgCPQ2TXOqFGjKqZMmZLnynr6/vvv0wBWrVqV3KdPn+q//vWve8aMGbN/yZIlTT5XX0TM06K1ngceJf9MH+c8DDzsoX2mr/MEobkz/ZdtDOuRQ4esVJ4OMlX5sxW7/HcCfm+ewV8tbzO2ejJZrY6kZ/sMSvfsoIJ0rM24OHaeqvdGpVLNpeY5FNi2gQnW22s4rkf4yRVjTAu4x/IO9/AOhVXvhD1euDR+Os5SIlp88cEHH2TffvvtDf5QJkyYUPbWW29ljxs3rvLss8+umDRpUrfRo0fvd3lHbr/99pILLrigsEePHn179OhR1bNnz6q2bds2WM9NT0/Xzz///Jbzzz+/h81mY+DAgYduu+22oDJBCgoK+ldWVppra2vVrFmz2sycOXPdUUcdVfXMM89svfrqq7tVVVWpUaNGVZx//vnlAH//+993nn322T26du2aU1BQUPPRRx9tbDzmLbfcsnfLli0p/fv376O1VtnZ2bUzZ87cCNCvX7+Df/jDH7ps2bIlddiwYRWXXXZZA9GzaNGitLvvvruTyWQiKSlJP/vss1u93WdaWpp+6qmnto4fP75nWlqa/ZhjjqmsrKw0A0yePHnHtdde26V3797Fdrtdde7cufrrr7/e8NZbb2VPnTq1XVJSks7Nza198MEHvWcWeEDpQLc7bUYMGTJEL1y4MNZmCIJPCu+aAcDVw7vxyrzNAFwwpBNTI1TRdEtq/arqlR2m85e0jyja/CYvWE/nEWvwcRBmbLSiigqiM1HmUM5LyVO4p/Z3nDb6VB77wpEl435fAHNt/Rhhdqw6zx03hxHJazlnagm/6KK6PslmEzUBFmEzYWdTan1oa1HV69RgadJv9QNj6XPf50HfVyg8NXFQg8rE40w/8VzyfxhbPRndvi+zbjkhpHGVUou01kOMstPF0qVLtwwcODC0wkMxwmq1UlNTo9LT0/XKlStTTj311KKNGzeucIma5sinn37aesqUKXlff/118EWcosjSpUtzBg4cWOjpWPN9vBKEBMElWICICZYMGsYQPrz3Jgqsjnj2HmpHQGNkUYkNE5Wkk0M5M1LuJpUaTqx+gv209j9AmAwzrWSQaQO/S/qcXZxKCjVMMn8KQIVOI1M5ltpcggVgxGenADA9BV6zjuHv1isASE4KXLS0p2FcUWsOsY/oxa94whXT0y2nFZv3HmzgaZHtEo3hwIEDphEjRhxRW1urtNY88cQTW5uzYEkUZO8hQYgBS34LOq4tLM4wO3YR/tI2CIAC6za4dBo/2vvQWjWNq2nMlgdPYmnqtTxheQ6Axy3Pkqf2k6UOsSR1UuQMdyMJR7yhBSt5B1axNvVKbrVMA+Cf1okN+tpRLLb3pCbFUcLiV3suVyXNqotJCdTD3J4yfkz9Y4O21ir2FX/dV4eSqaWTcqxIlNMKD8kkQgi0bdvWvmLFitVr165dtW7dulUXXHBBs492Hz9+/IF497L4Q0SLIMSAs575PqrXu7PzGkp0Fg9bL2GzPY9n2twBPU+hQqeTSQCT8Oa5AIw2L+Ity8OcYF7eqEPkH0AznOJqgnk+Zy29pq797e6Tedt2MlfW3AHAGntnPml9EWfXPMB34+fCvXt52nYWALcmTQvY2jxK+Tn1hrr3j9eeB8Drln/yhOUZTjY1DIBWynHO/Umv84blEQooIZNKWgfy+QZJ57bp9GyfwWMjk1mXegU3Jn2MVZs4hOFZtUZht9vtIqcEvzh/T7y6QWV5SBCizO6KqihfUZNZspDZ9v5s0vmMqnmC49LbcQNQQStaqwDqXf3fH+peDjc7kvfm24rZrnM4P+k7hptWMM/eP0L2O3DVIgGw2Kt5rPZ81urO5GeNALbyjf3IuiDZYZntoGQf2pwMZgtTbaO4yPw1uTg8XIE4WiYlfdrg/UadD0BX0x66soezzd8zvPrfbNPtATBt+pqfUm+s6/+d6WbMSlOh0xhQ/UrI923Cjh2Fe15DisXEnFtPhO8eq2u7sOZePOc+xAUrSkpKinNzc8tNJpMssQgesdvtqqSkJIv6UihNENEiCFHmmH98GdXrdaQUk62a3bq+2rdrCcHhaTno83wLVji0j9m2wfyl9mpeSn6cfTqTR6wTOdq0lvP5jreSH6Gw6m0iOWm6apEATBv8X56enwzAlT7WQ9yP7NOZFChH7St7AKrlWFP9dghL7d2ZYx/M47Xnka/2clHSNwC8ZJnCuJp/AhrzsnfdrtWaduqA026H2HrtqqFc9VpwG1BmUcnS1Gv5j/UcnrCeV9duUsDBvfD1PwAYUPUiFcRvGrvVav39rl27Xt61a1c/xMMveMcOrLBarb/31kFEiyAkEL87vhuvfu8I7DVj42rzTP5icUyms+z1SSF1ooV0MtVhzNiw4XmD13Y4qu9+ZR9ECW05q+bBumM2e/05bTlAWQTDQFtziBKdxdDqZ7kzsw+wxmtfTzqmTLemn2kLEJinJU+V8pb1ZP5qvbqu7UnbOaRQQ54qY5R5KX1Mv5FMLaeYFmFeOa2u3zf2gZxrnldvD3ay0ppmHPmju3Jkg/4paXoD0ZKydyX892LQNm6pua6BYPFUITXWHHXUUXuAM2Nth9D8EcUrCAZTa7NTfii4nZl9kZcZWJzC0MK23HdGMUvuGw3ANeYZdYLlYM4A5tvrt05xVU3d4/S+XGf+hKO6Ntl3DYBMZ+BphYcaIJt1R66puRWAf1hewajYlmvN/+ORpJfYknoxmVQ67ThMhU4nEG+OJ1FSSibZVAAa7cfOrBRFtqr0mCVUTTJX1d7JvbVXAnB70vsUmxxLbFvtjqWitfbOnFX9AP+svQhwCK5WyUmseXCsX9vdyfASJJ294AnHTf5uFh/ZRzQ4lohlLATBhYgWQTCYP76zmIEPfNGk/ZHPVtfVZgmGb24bFVA/1xO2Uooj1K/cZXmPam3hjtprWDf2XbSHP/c1dse2XrdZPuCRirt52vIfJpq/pFDtZJjJsazsCiStwPMed6t1FwDGmReQzz6PfXxhxtYgWDWFGv5ieZeJSV8D9bElrTnEAQ82+HIsuB8r1RmkKCutqMLuZ16/8shMFJoS7T21+U3bqQBcYP6Go9R6dGYnOjuzePaTwRLdkz3aseltb/UbnT8+j5RP/sAo02ICFXf3Jr3psT153yroMQq6hF/tVxCaEyJaBMFg3PeEceeFbzeFNF5asudlm8a45mezSTHUtBaAK2vvYKptFLakhpO9azL/RRfxvc3hgSk6vITx5p94xPIK36T8mXeS/8Fw0/K6FN8D2rNo2abbc1nNXQDcaXkvmFsD4FHLC/ySMskRO4PmJcuUBsdd95KpDjk9Lb6FijdKnUtXOarcT09ItzkCdku17/ozP9iKyVKHOM68CnvnY1iuuwEww+YQE+XOwntTUx4kfeePqOVTeS35US40f+PXhnkpN1Fkct8b1iF0MjmI+eAeaN3R43nxuDwkCEYhokUQIoTWmp827eOJ2euoDbCQmZ8R+b15ht8UWpOCYrWVam3hB3ux31H/av0d023Dqbr2B94dMafBsbPM39elRHvztAAstvekSluYYJ7vFB+Bc455HhZloz1ldFF7OMG8nNesY+hV9QYlOoujTWt5zfJPBpk21NngPi172iDQ07y9Q7cDoKMq9buEkl7rEC1lformXV97U91rW6+xXFVzB1fW3M5BHNupbNF5DfrrSY7U8UAK+nVSjgKyP9gcP8OTTIu5wjyLZanXoLQVep7s8TyRLEIiI6JFEMKk/FAtE575nq37GmbhTP9lOxe++CP/+XI9ve75LOzrHK3W8FfL2yxP9RxY75qoTUox2ryQ9bqAwV3aNjjmic26I7fWXo/OOYJDKTkNjp1lmsddzrgYb54WgErSedB6GeAIyA2FLHWQESZH/ZeptpHUkkSJc3lllHmpVxtSLE3/jXnSJNu1496ONa3yuzyUZnXUEduvfWfklJHJDTU38YntOOzF51BKJt/YB9Ud36A71b1ef/Ua6NCfHTo7gM+o3sAPbI6S/K8mP8bfLa8DsOecadDD8+7gEtEiJDIiWgQhTD5bsZOlv+3n2a8b7lu2dnfYm702oKupfofhHMq53DyLq80zyafhli5q/1ZyVQUz7aHFO4ytnsxqexeGVT3J1/YjyVelYE6pW+rwxl5n/Ed7Veaznzey1QGuS/qEUp1RF/83mrYAACAASURBVCdzRc0d/KX2ag5qRzCyK6bFXYTdOKqn1zHd+7lEy81J0+uCe72RbnMsIZX5ES0AM+zHclPtH70qw8Kqt+le9RY62TFWvirl/KTvfI6ZRjUAb1lPZoMuaHDsyKoXqO54jNdzxdMiJDIiWgQhTFxP7aYI/jXlUcqjlhfr3i9MvY4HLK9zr+UtPkj5O1C/TGIqcdQX2UDnuv6NPQ/e4h4UsEZ3YVzNZHaQwzW1t3Fy9aNw/Q8eNwl051dnkbVPU/5KF7XbZ19PDDOtpJPay1f2wbim3hLa8o7tZJ6xTgDA7vyX5b4k1CqlaeUGT7fnvpN1jvJdkT3V6WnxtzwUGKrObndONC1t8P500490dAYyt3Pat0J3Y5nuzpTa+nTn/bT2mf0kIS1CIiOiRRDCxFWorLEQ2FTiu2hbMFyZNMvrsQK1j+5uMRLmckf67UmnnBbUNZTyvLSwURdAux5+z9/qFr9xW9LUgK6ZQk3d6+Od2UqvWMc16XeIVICA42X8Zf32V76DotOs5ZCUSjXJAV0vUNx/R15P/idmbIAjuPaZ5Cf5IfWPPG95gt7qNwAO6VRA8azNIdrur73C7zUk41lIZES0CEKYuII6TY2ecOesDt7b4BFrNaclOfa5ubbmlrrmk6sf5Zaa6wDopzbXx7Ts3wrJGVx0Yn1sReOn70g8jB8ilW9tAwAY4sz48Ucbt2WagSaHkNimc5v0m2M/ivm2YqY54zsC9SY0DtJ9y+oIXm2lqn2el2atgLTswC7igXMHd2rS5rLkidpz69oeSnoVgOcs/65rG2tewACTY6lxiXaIRRtmCqve4XXbGMC3MBFPi5DIiGgRhDDYsOcANuf60Fs//hqZi6ydSVd28I71JL6wD2WlvSvvW0eyURfwpX0QNdrMRWZHTRN+egF+et6RDus2e4Xz9P3UxEH+Ozm5ovZOwBG3EQhZqqE3yqaVx1os23QuF9f+ldW6a8C2eOIh66WO6/rZuiDNVgFpnovtecNdLJx4RFPh5eJ52xn823oOAH1Mjt+Z4537Od1juhmAI5VDtOz1UifG149TRIuQyEgZf0EIkeXbyjnj6XkN2mz+0lKC5J3fHwML/4RdWeom3NNrHqk7XkEGn9qP4xzzPD6s/Q2WOZdlhvwOCL5mR+NUYLNJccbA/CBGUMyxDaKvKYBNGGnoaQFYqI8I7Cp+7svb4SqSqdaWJmKpMWnWcsgK3dPiy7pqkvm39Tz6qc3kuQUtH9Ip/KwcnirXLtquZbHG+ErZluUhIZERT4sghMDd05fz5o9bmrT3+MtMQ68zrHMqrPmUiqwirxPYdJujjPs/Sm6A7Qth0KVw3PVAbEq679A5dFSlXGqe7bdvG9VQtMyyDTXEBu+3rUhRtVxt9v1zSrOWQ1obQ2zxxkHS6Kl2ABqrNvGqbSxlKot/1E4EYIndexxRqqW+4OAHfzguonYKQjwhnhZBCIF3f47QUlBjShyxITs7nw6eC+0yz96PN6yjOTl1LQU9+sHwW0O+nBHVVJfaewCzecjyGh/ZhtcVWvNEY4/HOt00FiQsvNyORdl8nhZuTEsgpFJDmqrh0+R7SFJ2KnUaSsGLtjN40XYGnhaBOmSmMvnc/uS3qf9Me3domOEky0NCIiOeFkGIU4ablsM7F0BSGnvzhvvoqbjPehV3dngZJr7bINMn3OWhUJhur7e1k3MvHm+4YkvesZ7EbNtg5tn7BXQNf3fl67ZftY7Fqk14jwzRZNTuCzqmxR+NbfqvM6jWtfN0ZRNx1/QmWqWYGXlE+wZtrVMtdM+pr6HjqUKwICQKIloEIQ45KmkzbyZPhqQU+P1sKrOKQhonFstDGlPdzs/Hmlb77NtGVWLVJv5ivZpram8j0Lwmf1rM121v07kkKXuTeBoXrvL5WLx7iIxglT34oGJvItRiln/lQstAftMFIQ65NHMxCg1X/A869A/onEB3O44GP9t7A1Cktvns11PtcFbaDc7AQHt76rdLOzwoHbxU7i1wVRjOHxykTcHdQwXp7ND1S1D+MpoEQRDRIggBY7Nrqmp9x0IYRXubM4AluzsQ/n4yTSvihjmgH8rJoEKnU+MnbC5HlXNQew4wDgdf9+cq59/dy6aFdTVm2nSJyPVdaEwMq36KAVUv8bFtGO/YPO8lFAjuFXIlpkVIZCQQVxD8sKeiiqP/8SXZrZIpPVjD8r+dasi4rThMnipjk85v0l5cuxKKxoY9AxkRWBsqZTqjSXZQYzqo0jqvTDCEc18rdSEVOp1nk5/kTesq3radwhpdL1D6uNK123QGNnoexJ99QfiCKmjFn2pvBCDHgNU8SXkWEhnxtAiCH37Y5NgPpvSgo+T8HdOWGTCqZnnK7/kq5Ta2pF7Mn8wfAnBP0lvMSrmTtvYyGH6LnzECuEoMZ7D9ZNDWx8aEp5l+pJPaSyuqDL+2r9u2ksTttZMAuCxpDu8mP9TgeCaH2JVeBMm+N4gMnvAFpLcRJPhWaCmIp0UQPGCza75cvZvRxXlNjn22wkvucRAMN63ApOpn1lssH3KMaTXDzKsA2GvKIadLaLs0B0I0prj9fjwtzyY/CcACe2AF5dwJuIy/l46z7EPZp1vTTh2gbSMbs9UBDiQX0CFoq4K3L9jzApGgsjwkJDIR87QopTorpb5WSq1SSq1USv3J2Z6tlJqtlFrv/N7W2a6UUk8qpTYopZYppQa7jXWFs/96pZT/HcMEIUxe+34z1765iE+W7uC5b0JbImiMBSt/Mn/I4pRrucH8cZPjLsECcEgZ85Qfy+Wh/WR4zdApwJEKvcWex8u24DZ2DIRAbnu/zvDYnqfKOGgJr0ZLLJdoRLMIiUwkl4eswJ+11sXAscANSqli4C7gS611L+BL53uAcUAv59e1wHPgEDnA/cAxwNHA/S6hIwiRYle5Y8lid0UVa3YdMGTM5y1PcIvlQ9qqSo4zr2KWbQiFVe8wtOrZJn0fy7o76PHDFShKGTvZlukM2irPn107VQHAg9ZLCWWa9XdGIPexTHdv0pZCDbmqnPKUYLYuiB4iSISWTsREi9Z6p9b6F+frA8BqoACYALzu7PY6cJbz9QTgDe3gR6CNUqojMAaYrbUu1VqXAbOBsZGyWxDAsecOwD9mrglrnFzKuD3pPa40f87J5sUNjt1XeyUAJbRhcNXzXFNzK7t0W86ofohtScFnrsTbhLafDLLUIUzYmxxzVcL15u3wS4ACzVevP9dex6c2xxJcJ7UHgHY4xFQonhZ3k2SJRhAiQ1RiWpRShcAg4CcgT2u903loF+AKGigAfnM7bZuzzVt742tci8NDQ5cuoacqCgKAyRT+rNNN7eTrlD83ab+w+l4qSWU39RNjKZnMtg9hdvUQABpXCDF6ucGbV8bIydYlSLKopIzMBsd6qu0A7CYyTtNA7sOOiRes4xlv/pGzTN/ztO1shpocIrXWZHxhuUgKGR12UrwgNA8inj2klMoAPgRu1lpXuB/TjtQGQ/7atNYvaq2HaK2H5OZ63xZeEALBAM3iUbCU63R+0n1YqbuFf4EIYPTyEECbRvsL9VWbud/yJgDbdPsm5wWCUfP/ct2dMp1BN5PjOaqPybGn1P7U8JaHYupoETePkMBEVLQopSw4BMvbWuvpzubdzmUfnN/3ONu3A53dTu/kbPPWLggRwxTmP/7jTcvrXo+snsIF1fcCMNceWHXbxgRiTjAmRyMVuhyHaGlLw7iWS8xzwh7byHnZjuJc8zzaU8ZZ5u+p1hZ2p4e2bYKLSOkGb+NKyrPQUohk9pACXgFWa60fdzv0CeDKALoC+Nit/XJnFtGxQLlzGWkWcKpSqq0zAPdUZ5sgRIztZYfDOv9y82xKdQYDql5ii+7Iz7o3J1c/yo21fwppvEA0RjDTVqBBu97ETYdM/1Vs6z0tDTOIOjs3UVxmj7y3KZDbfNdZifbn1BvooMqYbhuO3ZQSYctCQwrHCS2dSHpajgcuA05SSi1xfp0GTAZGK6XWA6c43wPMBDYBG4CXgOsBtNalwIPAAufXA842QYgY+w/XhnTeV38+kUvNsxljXshs2xAqcKUuKzbqJqFYcYeRsRFltAZggnl+g/bOqoT/2Y5lQs2DIY9tpGfhMeuFPF57HgBV2hKRFOxI06CMfwztEIRIE7FAXK31PLz//Zzsob8GbvAy1qvAq8ZZJwi+sdpDm7y752ZwhfkLAP5mvdxIkyKO0UsMu5ybAU4wz2eCeT631U7iR3sfOqp9fGEfgo7gM1OwHoknbefwpO0ckqmlBgujQ7imiAVBiDxSEVcQPDC6T3u+W1cS/Im//kgv03ZesJ7OYYzbCDBcD0jjs8MtBx/IsksNlgbvH7O8UPc65FTnIK4PwQsxl82RiEkxYkiJaRFaOrL3kCB4ICs9ObQTP7uDgzqFN6zGbKoYDEZUvzU6dXZy7UUe23fpyNaHjH0CTewMiP29C0LkENEiCB4INOX5kxuP58EzirjcPIt2lMPOpbxrO4ntGJt2H8iTtK8esZrHnredyenVDzdoq9ZJfGQfHiOLfJOebDZkHBEOghAZZHlIEDxgDnDWSTFpzls2iTTLIh6wOAo9z7f3NdyeQDwgvnqE6j8xwnuzUnfjpOrH6KW2kakO8Y3tyIjGs4TD9SN78NgX62JthiAIXhDRIggeCHSybr/ocdJ2L6pvKDiKrzYOipBVwROs5GgcwGpUPZdNOp9N2rj9fCLlyIjlBpOCIPgnPh93BCHGfL1mj98+Fqy0+eVpDnY6sb7x1IeIpzwSb5Ij3Lk51nfYHMuVBCKIDAnWNWAMQYhXxNMiCI1Ysb2c9xf+5rdfZ7UHpe1U9DyDsRvPIoMqPus6DJgReSM9EG5Mi1KJE4sRroMolI/BmEBoQRB8IaJFEBpRdqgmoH5jTT8DUJV/LL/pXyNpUkAYEdPSXCquxru2inf7BKG5IstDgtCIwArLaS4wf4u18zBqswojbZIQJKE4PY7plu2/U8DXj4xsCSiLLFHcZYLgAREtgtAIewCiZahaS6FpN2rQZVGwKHxkGvPPq1cOjbUJIWN0fR1BiFdEtAhCI2wBiJYhJkdarLn49EibA4S/bKOB7jmtAuoXCIn4NN8qxbjVck+fjr9PbHRxnohLQfCDiBZBaMSf3ltS99rb3NzHtJXKlA6QmhUlq/zjb6L87OYRfno3HyKlmYxK8Q6FE4pCL0goZfyFloKIFkFwY8+BKg7X2vz2G6Q2sCerPxA/07+n6da9LSXJf7XXeLmXWNOcHUnN2HRB8IuIFkFwo6rG7rdPL7WNzqYSdrc5MgoWRQ+FpNyGgz+x4O+zDefzl5gWoaUgokUQfOBpIhpjWgDAxryx0TUmBBL9qfusI42rsisIQvwjdVoEwUnhXf6LwmVRyW2WD6jWSZSb2kTBqugSyzotWWkWfrl3NK/O28zDM1f77KtQrH5gLMlJ8fnc5Wl5yZ+AVCqwPh7b3c5szktbguCP+PyLF4QoE0iaM8Cl5jkALLIXsWJ7eSRNakDoywaeideJzWxSXHNC94D6piWbMQe6HXeU8fT5ygKOIISPiBZBAH7aXOqxvXFqb6Y6CMDltXfRJt3i7BNZ24To0VwqAgtCS0VEiyAA9gBnqzxVxlZ7e6wkcWpxhwhbVU+ouqg56KneHVoD8SX+mnMKcXO2XRD8IaJFEIKgUO1mh86hdWoSo3q3j9p1E9UBsGXy6Zw/pDMQpJcjDudld9EVqnAYUtgWgEknBrZEJggtDREtghAEBaqEzboDyebI/ulEuuKsx0J0QVzTSPPiUH+ETagpyK5aOkfktQ794on4gQqCExEtguAD9///Zmy04wAlRD5rKFaVWRtPtif1zouJHc2dUDwtsqwjCP6RlGdBCJDhphWYlOaATmvkaYj8ZBMLEXP7mCO4eni3qF0vnmJawsZT9pCfn6EUiBME/4inRRAC5OakDwEw479qbrwRiubJb5NKqsV/6X+jiIfMnTgwQRAEH4hoEQQfuD/9L7H3AOBl22kxsCOR3BANicdbi4RN/sv4x+EHIQhxhogWQfCB+9N/J1XCantnbATmfRjWo12ErAqeUCZhX5NoPAqNeCLcjyccr5P8aIRERkSLIPjAfXLupPayTee6jgR1brgYHdMSj6IjGJsad33798cYaouL84/qFHBff96wUEv0C4JQT8REi1LqVaXUHqXUCre2vymltiullji/TnM7drdSaoNSaq1Saoxb+1hn2wal1F2RslcQPOHubchXe9mh48d7EgyBaB4VYL9IEc61j++ZY5whbhgqPKN0XRE/QiITSU/LfwFP2+A+obU+0vk1E0ApVQxcBPR1nvOsUsqslDIDzwDjgGJgorOvIESVVhwmSx1ih246OcokER7y8QmCECgBixal1HCl1FXO17lKKZ+5kFrr7wDPG7o0ZQLwnta6Wmu9GdgAHO382qC13qS1rgHec/YVBEPZW1nt83hHtQ+AnVHytBgdeNschFU82GiUpymRA6cFIZYEJFqUUvcDdwJ3O5sswFshXvNGpdQy5/JRW2dbAfCbW59tzjZv7Z5svFYptVAptbCkpCRE04SWykMzVns+4Jx7CpyiZXuURIvRMSxGL/vEOtMlUFEQau0T1+jxkIYtCEI9gXpazgbOBA4CaK13AKHUmX4O6AEcCewEpoQwhke01i9qrYdorYfk5ub6P0EQ3PA3OTX2tDSXB+nmYicklkCI5ccea0EpCJEk0Iq4NVprrZTSAEqpVqFcTGu92/VaKfUS8Knz7Xags1vXTs42fLQLQtTIV3uxacVu2vrvHEd4EwLxNLFFcikl3PuMpuhzv1QiCThBMJJAPS1TlVIvAG2UUtcAc4CXgr2YUqqj29uzAVdm0SfARUqpFGesTC/gZ2AB0Esp1U0plYwjWPeTYK8rCOGSr0rZRbbHGi2RmNdCmchjtV+RUcSjV8jIj7SZ/3gEIS4IyNOitX5MKTUaqACOAO7TWs/2dY5S6l1gJJCjlNoG3A+MVEodiSP7bwswyTn+SqXUVGAVYAVu0FrbnOPcCMwCzMCrWuuVwd6kIISKax7NZ2/QQbjx4M3wJgQ8tcdaNAQzqQdqaqz28wn3s/T+c4tufSBBiDf8ihZn2vEcrfUowKdQcUdrPdFD8ys++j8MPOyhfSYwM9DrCoKRuKa8jmofy3X3uvZIzwuheE08TWjehgn3qd/IiTGeJtnGIkecI4IQX/hdHnJ6POxKqawo2CMIcYUCFHbyVWmzLSwH8SUMoknI3q6W+oEJQpwTaCBuJbBcKTUbZwYRgNb6pohYJQhxglLQjgOkqNpmLVoae1ZSLPG3g0dkNimM0fJQCGLJqPvPyUgxZiBBiEMCFS3TnV+C0KLQuj7d2VM1XPAeZ3DTyb2Yt2FvSNdtPGYgyzmelpS8TYR5makBj+GNnIwUv0X5giGomJZm6QiJjoC6dXRRVK4jCLEgoMctrfXrwLvAIufXO842QUh48pVDeLgH4gYyaR7dLZu/n9k3pGsalQnkPeU5gHP9TLI9ckOqfBCSLY0J9OMJNxg6Vhk/4Vw3OSn+vGiCYBSBVsQdCazHsQ/Qs8A6pdQJEbRLEKKKN4+BUpAf5Wq4oRLN0vHN0tGRwEy5YGDd6+bphRKEwAh0eWgKcKrWei2AUqoIh+flqEgZJgjxgEKRr/ZxWCezn4xYmxM0wUxgjUWPPy+F0ZNjcLYG1i/YmJZ4qaUS7GfbryCL3NYplBwwbrlOEOKRQP2IFpdgAdBar8Ox/5AgJDz5aq8zCDd6Bd9aYnG5eDI/Xp0VvuyKV5sFwUgC9bQsVEq9TP0miZcACyNjkiDEF/mqlO1egnAhvieLeBICXonjMv7RJOz4G4PsEIR4JlBPy3U4qtXe5Pxa5WwThITGEdMSfDXccDGquFzw13V7HeVpMB5Tno38DKIlIJuTUBOEYAnU05IE/Edr/TjUVcmVYgBCwmPBSi7lzbZGi1ch0MznNQk2FYSWSaCeli+BNLf3aTg2TRSEZs9z32z0eqw9pZiUZgcNRUs8Ps0G4p0Z169DwOP5DcQ1+DOIhCciHn9OgRDKZ9EslgIFIUwCFS2pWutK1xvn6/TImCQI0eWfn6/xeqwjjhotoXpaQp1HjEpfPmNgPgBdsh1/rgM6tTFkXCPTqyMpK4LOHoqQHdFEvFBCIhPo8tBBpdRgrfUvAEqpIcDhyJklCPFBR3xXw4XoTBKhxlZcdmxXLhzamZQks8EWGY+hmzCGKYXqbImminEzWYSHIHgmUNFyM/CBUmqH831H4MLImCQI8YPL07JTZ0f1ukYF4iqlGgiWWO3FYzT+REms79OT6PBrUWL8aAQhovhcHlJKDVVKddBaLwB6A+8DtcDnwOYo2CcIMaWD2sc+3ZqqRnHnzf1J2NukH8y8afRnEIxOy26VHFC/WMW0RCq+JJDPvJn/agqCT/zFtLwA1DhfHwf8BUcp/zLgxQjaJQhxQT7RT3eGllVcLhTxc0JRbkD9Yu1xCQpRG4LgF3/LQ2atdanz9YXAi1rrD4EPlVJLImuaIMSeDuxjiw5sgmxJRGJ+be7eq2jgW5c2I4EmCCHiz9NiVkq5hM3JwFduxwKNhxGEZksH9vrNHPK1BBGq8yMQr8lH1w8LbfA4JS5SnhsZEaM43PA+CxF/QgLjT3i8C3yrlNqLI1toLoBSqidQHmHbBCGmZHCITHXIo2iJ9rzgaRIb1KVtg/eBLCnF4wpSJONOQl0eitf6LuKNElo6PkWL1vphpdSXOLKFvtD1j38m4I+RNk4QYklH5VgZ9ZXuHE8EE9NixOQXzxNoPIqOYARjPH+2ghBL/C7xaK1/9NC2LjLmCEL8UKDCKywXDkYWbwuGWHpijLzlWAfgxlJ0xKNgEwSjCLQiriC0ODoqR2G5WGQPxT0RmBf3H6o1fMywd04OUcXFMuVZEBIZCaYVWixaazaWHPR6PF/tw6pN7Kat1z4Qod2J4zH4JEJEciKOtcclmrSgXxmhBSOiRWixvPr9Fh78dJXX4/lqL3vIxu7BIRno8o3MI82LWP68jAqkFm+MkMjI8pDQYlny236fx/MpZRexWRqKdExLIKP781I0h9iJUG2MiPfMz+fp7l0Tr4kgeEZEiyB4IV/tZSeRyxxqnRK4ozNak1iiLack2v0EIqZE8AiJjIgWocXi6/+/wk4HFTtPS6QIKi3axydkpCMiItV1DRrVyPnfn03u3jVZ4hEEz0RMtCilXlVK7VFKrXBry1ZKzVZKrXd+b+tsV0qpJ5VSG5RSy5RSg93OucLZf71S6opI2Su0PHxNDDlUkKKsEfW0BENAG+XJTFdHrD0snnd5jo5N8msgJDKR9LT8FxjbqO0u4EutdS/gS+d7gHFAL+fXtcBz4BA5wP3AMcDRwP0uoSMIkSTfWaNlV5iixagsoECGCeRaImyCY3RxXkjnhfJjl5+MIPgnYqJFa/0dUNqoeQLwuvP168BZbu1vaAc/Am2UUh2BMcBsrXWp1roMmE1TISQIhtPftBkgbjwtRmFkKnU8659Ql4cafzzjB+QbYI1x+NznKop2CEKsiHZMS57Weqfz9S7A9RhTAPzm1m+bs81bexOUUtcqpRYqpRaWlJQYa7WQMOypqGLuesfvh69pbbBpPZU6lQ3a469bsyeeBEcsMnX8EUcfTwMCua94tV0QjCBmgbjOfYwMezjQWr+otR6itR6Sm5tr1LBCgnHOc/O57JWf/fbroXbwi70XVi+ljCI96Tf2iBj1h+LP0RKoI6a5LDM1h7RsT4TjEBOPi5DIRFu07HYu++D8vsfZvh3o7Navk7PNW7sghMS2ssMA2Ozax8Sr6aF2sFHnN5vJOVia62QeLKF6XCIx8UsqsiCET7RFyyeAKwPoCuBjt/bLnVlExwLlzmWkWcCpSqm2zgDcU51tguCXvZXVjH9qLtvKDjU5Vmuze5228ygjQ1WxUedjCihrJzw7PY/ZcFCjLxHr7JpI09xFmbffqUDuq3nfuSD4JpIpz+8CPwBHKKW2KaWuBiYDo5VS64FTnO8BZgKbgA3AS8D1AFrrUuBBYIHz6wFnmyD45aNftrNiewWvfb+lybFam53piz077XqYdgA4REsgqiUKJLLEiITAiHVMSyhCNkGdeoJgKBHbe0hrPdHLoZM99NXADV7GeRV41UDThBhRY7WzaW8lvTtkRuV6dqc/vrLK2uRYrc37pNZDOUWLPR9Tgs4kRgiFdq2SDbBEMIqWtMmm0HKRirhC1Pjb/1Yy9t9z2Vl+OCrXszn/ib+/8Dcqqx3Cxez0nFhtdq/ndVc7qdSp7KFNREVLLKYYo66pgAfP6mfQaMYTcsqzQZ9QLPVDosZhCQLILs9CFFm0pQyA8sO1dMxKi/j17Pb6mWPoQ3M4XGure3/0P770et5p5p8o1a0BFVBMS3PE27wWzFzbOtVCccdMVu2sCNOY8E73RNjLQwn6cxeE5o54WoSoEYngz3d++pXCu2ZQbbU1OXbAbVnIXbD4woKVPLWfLiZHLRezF9XiPqmF+mQby3kxHE/AGQPz+c9Fg0I+f8ZNw0O/eJAYVWTOCIIZU1Z6BMEz4mkRmjWPz14LQMVhK7mtzXXti38t44XvNgU9niue5RnrmQAJG9MSDk9NrBcsocytffOzjDPGD80pS0p+1QTBP+JpEaJGNNNQ56zeHdJ5RWobAJ/YhgGQYgnvT8SoJ2ajgywTfYJs/LvWq31GcOeH+flE6vxE/7kJgj9EtAgJSahzfEe1D4DtOocv/3wi6clmP2dEhquOLwz6nKuHdzfekCgQiXm4sYfl/UnHReAqwVGYkx5rEwSh2SOiRUhIQvVLdFClHNBpVJJOj1zvT+eR9hoFu1HfhCPzGd7L/+aOLTVWIjvA9OxIfj7pyZFdjW+hP1qhhSGiRYga0YwvsIc4++SpMnbrtkGdY7R8OaVPe4NHbIpXmxNE6Ncb/wAAHEJJREFU1YQrKsNNG45pynPsLi0IEUcCcYWosW53ZfQuFvLyUCk7dbaxtgTB7FtO8Onh8YZMVA1pTgG4Lozy3jW/OxeEwBFPi9Cs8fZEG+o2eQVqL7uJnWjpldc6brYOsJj9/3swIkBYiqE1JUEcXoJgOCJahITgUI2Vl+duqqt0G8pkmslB2qv9rLV3qmu7b3xfj30DnWej9cQfiYl/bL8Oho+ZkRId525z3zAxHFrunQstAREtQkJw07uLeWjGao56aA4/bdrHS3M3Bz1GvnLsxblD1we0Ht0tmxOKcg2zM1T8SZ9rTwgscygYEeWtsF44rPj7GMPHjEdinTItCImKiBYhIVi6rRxwbBFw4Ys/hjSGK9052JiWePDk9+kY5CaUXmbFeLgXIwjVwxXv9+9Ly8iSktASENEiNGuMfCLNd4qWHbpdUOdFY3fdRH7wbm731iO3VZO2pfefGgNLPCNeGiGREdEiNGuM1Asd1T6s2sQeGqY8+xMl0XjCNeoSVx3fjTMG5nP18d0MGjE+iWRMy6XHdmXBPac0aMtKszR4H8rvhIgNQfCPpDwLgpNOqoRdZGMPQMsHOr/Em8s+K83SYO+gRCXSAdC5rVMiOr4gCJ4RT4sgOOmi9rDVnhf0efEmTKJJS773SCKfqyB4RkSLIAAWrAw2bcAWwp9EVNKaE3gSi8SySEtOeRaEREZEiyAAg9R6AArU3qDPTaSn4kS6l1Aw6v4bC7FhPYIL7g5FyEUjIFwQYo3EtAgtlnz2coB0xpgX0FmVAHBz7Q0BnetezM2oqSLJV10UcRwERbjeL6O9P9cEWEfHCMTLJCQyIlqEFsv81JuatK3TnTz09I1RT7iDu/rYqDHKD9FDC4PbNDIcJGsmCOTDElo4sjwktEiyqWjS9pO9N9UkBz2WLz3RXB32J/SKfRXgcIiktyFSqzChbJQpCC0N8bQILY6UJBMD7RubtE+unRjSeC0llODda45l/6GaBm3xuptyrO0K9nfiX+cOoF9BlvEDC0KCIaJFaHEc3S2b/+Rbsf5oYmD1S6RSQy1JVNC00qk3Gj7Ht4yJ5Lggg0mFevz5ffKyUqNihyA0d2R5SGiRZJYsYa3uzEHS2EdWUIIFIMetuFgiPfzGwkPhbynn6uHBV+8NdXkokPuPTIp2oB0lpkVo2YhoEVocStth+yIW23uGdH6X7HSeu2Rw3fsE0ix1BDo3ugTbvy880tAqsWcMzK97HYtYj3BjYoLVFsf3zPHfyQ+J+HsoCI0R0SK0ODpaf4PqChbbewXUv7En5fyjOtEuIzE9LaFSnB/kLtN+eGriIC4a2hmIX+fCG7872pBxijtmYvaV7h4scfp5CYIRxES0KKW2KKWWK6WWKKUWOtuylVKzlVLrnd/bOtuVUupJpdQGpdQypdRg36MLgm/6Vi0GYInuEdL5jSfRxksKk07ozl9P7+M4ZpCgidayzagj2gMwPITsoUSYKycMLABgdLH37Rxc93lCUeCfkTJIeSXCZywI4RDLQNxRWmv38qN3AV9qrScrpe5yvr8TGAf0cn4dAzzn/C4IIXFk1U+QlMqmqo4B9W883zSegBoLk7tP6xOOeTFlSGE2WyafHtVrepvPXd4HI50Q/ijOz/R7/0YJEEEQgieelocmAK87X78OnOXW/oZ28CPQRikV2GwjCB7oVLsFuo9CB/jr31iUNPG0tODlIe3ltRHcMaY3Vw4r5KxBBUGf29xSngMeNzLDCkKzIVaiRQNfKKUWKaWudbblaa13Ol/vAlz+2QLgN7dztznbBCFo2lNGtm0fdD/RsDFjPUHGC0b7H7LSLfztzL6kJJkNHjn6iG9GEIwhVqJluNZ6MI6lnxuUUie4H9SOuuhBzQRKqWuVUguVUgtLSkoMNFVIJPqbNjle5A8K+JzuuQ3ToRtnlpiCWC4Y2CmAAmIJSnpy6OJjQJCfWyQq4vbu0NoxtsFDBzOez66inYUWQExEi9Z6u/P7HuAj4Ghgt2vZx/l9j7P7dqCz2+mdnG2Nx3xRaz1Eaz0kN7d5lyAXIscA02bsmKBD/4DPuef0hjEqjWMsXJOZJxp7YRJ1XlH4vrcTi3JZ9cDYsMaPNUOc+zEFYkssw14k5EZIZKIuWpRSrZRSrV2vgVOBFcAnwBXOblcAHztffwJc7swiOhYod1tGEoSAudo8gz8lTWenpTMkB15MLiXJTPec+v7+AnN9Eep84i1GYsZNw3n47H4hjmos7Vp53repoE0aT18cuGfLIwk8EyfwrQmC4cTC05IHzFNKLQV+BmZorT8HJgOjlVLrgVOc7wFmApuADcBLwPXRN1mIVwL1XORRyr2WtwHYbw6vHH04Sw99A9lfJpjx8rO45Jiuho4ZDO47XN8+5ogGx0b0chRMO6Z7Nq1TLVG1KxKEE1wbDWHSt8BRKycpmulWghBlop7yrLXeBAz00L4PONlDuwZuiIJpQoKisPNC8hMArLR35fmcu3gq+EHqX4YxJyTqfKIUTQJmJxxZwNz1e72c0fyo0ywxdI34uvSLlw9h3a4DpCfLlnJC4hJPKc+CEDSBTB9jTAs50rSRgzqF02se4YC5bfAXMigYxeWl6dou3ZgBY0xepmOjP4u56b+SojxH+f1ju/n2bAWyvPbX0/s0CYj2RSQzugL5nYtYyrOPcTNTLQwpzI7MhQUhThDRIiQ0Zmz8zeIo/3NBzX2GjBlucbG1D41lzq3GpVzHkqcvHsxj5w+ka7tWTbwAAzq14ee/nMz5QzqFfZ2hhdl89eeRYY8TDi25Ho8gxAsiWoSEZqDaSAdVxsO1F7NSB79bsAv3+SoYyeJpoktJMjfwTLx0+ZCg7elr8F4/oZLdKpnzjnKIEk/32j4zNSYVZAOJO/rkxuNDGzuA22kSrO3DnkikZwtCoiKiRWjW+Hv4HW/+EYBPbMMMu2Ys4lJc93nhkM7Mu3MUUycdF30jIkSspuwkk4kHJ/QN4ozYu1ok00ho6YhoERKavqYtLLH3YDfhrfW7zxWx3numU9t0WqXEX7Blc5xQLzuuMOhzxDMiCLFDRIuQwGh6q19ZZQ8/JTjUZ+z+BqU4N4dp0hykCyrWmVTBiqxIxbQ0R7EnCLFCRIuQsORRRpY6xBrd2X/nIAhmkjmhKJcf7j4ppHObG0cXZvPHk3oG1HfxvaNZcv+pEbbIN6GKkFB+hon8cxeEaBJ/PmZBCAJfc0Fvk2OfzbX2LkGN+dLlQyhok+b9mkHOQB2zvI8VKLGPpvCPyaT486lH8NRXG/z2betWPTfWE/pzlwymV573rRgEQYgfRLQIzRpfk/npJkcQbrCeltHFeT6Px3KOjfUEn0i4Pstx/TsG1F9SngUh9ohoERIUzSnmRXxrG0A5GYaO3Kdj06fyd645hk5tgisY99ylR/Hi3E20Sgl99+N45PELBrJyR0XMrp+TkRJQv2BFiKtgXSi6UbSmIBiDxLQICUk3tYtsVcln9qMNGc+1x8571x7LUV2bZiIN65FDlyCr3A7vlcMbvzsaUwDukyM7twHg5D6+vUDxwDmDO3Hv+OKYXX/enaMiMq5L5Bjt7QpmOBE/QktHPC1CQjJYrQdgkb3I0HFdZeuN5AhnPMXVw70Xv+vTMZMND48jyUO5/ObOgSqroeOlWsyMH9CReRv20i0n8NL/giDEPyJahITkKNM6KnQ6G3R+rE1pgKcn5batktky+XS/5yaiYIkUFw7tzNmDC5ps4hiXeHHdvHDZUZQdrOGu6cujbJAgxC/yX1BISAaZ1rPY3hNt0K/4mH4dAGiTZjFkPKGeSNRrUUoZLlhcITAhFZcL4ZQxfTtw0dHBZb4JQqIjnhYh4eimdtLH9Buf1xoTzwJwx5je/OGEHg1SdQVjaC5ZObpetXhl+vXDKK2siYo9gtASEdEiJBT/3969h0dVnwkc/74zySQxF0iIoRDuVxEM1yoiaFG52xVXuuq2q9tiXdryqOvqo1a2hUe77brWrbeWuhbX1q5YtaVUvKDWekOuAoGgkoAoYJT7PRcy8+4f55dkEiZXIDOHvJ/nmWfO/ObMzHtOzjPz5nedEljBr0IPAbBaW96f5eYGJkcLBsQSltPEJzlLs4zokQ3Atj1H4xyJMWcmax4yZ4wC2VKTsPzH8etYFmnJYnieK4YmVh+Y9iDil6qWFuiVm87KH15GQbdTs4yDMcZjNS3G1/Yd9ariu7KHxSn/DsCdx7/Ls+HWDXvt2shMuOb08EvO0tJ5WvKyUmv2bawfjA1jNqb5LGkxZ4RRgc0AfK/ylibnZkltoINm0bxJCbl68pnuneLd8Q6hWXrkePPw5GY2b/K60yHeK4wbE2/2DW1865E3imu2hweKKdMQSyOjqP7fNT0U5Ghl+ITXpYW8pOWBbwzl9ufW15S3RcJiPzon2lTaNrPnZqUmcegk5oSZPb4fBd06MH5gXk3ZA98YSlZqy66b+6YPIaLKj/5cBNjSDMa0hPVpMW2uVUNGgd2HK/i7R9+l111LeOKdrfz8tc01zw0PlFCofQhTW4vy5LfP56N7J5NRLxkJuflOzvmKN6nbdy7qzeb7prQqpuaaPNgbMn12HP9LT1SRSNt8zuu3XcJtE2o7Z1cnr80xrn8uScEAl55Td0biGSO7MdH9bWOpDHtNSqGk2mv+W6N7cv2FvZr92caYWlbTYtqcRo0Xmf7Ye+zYX8bqOZc3uP97JXv45hMr6pTdt+RDAFKo5IrAcoYFtjC/6oo6+wQDQmpykI3zJrHrcDmPv7WViqoIt08aCMCQ/A48P+tChnbvSPJpnrjtymFdeaXoC87tknVaP6c96NTKUVx5Wal1OsY2d7bc9T+a2KIEJ1pZpVezk5bc8Fft3VMGNfv9pgxpOEEypj2wpMXExbvFewglBVi3/QAAy0r2MKZfLgAHjlUSUchJD/Fi4efM/r+1J7w+xHHGBjZwU9ISRge8BGZpeFSdfaKr3fMyU5kTYz2cUb1OXEfodJhyXhfevmN8i9cnag+0BT1x375jPFlprf/aurj/2SQHhfumD2n2azqc1foJBY+55snGFsU8v3fzrsF37xxPvnUUN+2cJS2mzV312DLKjtfta/LCBzvJy0qhrPRjfv3sIl6MjGZaQVeWFJa6PZS+8jljAkWMDWxkTKCITCnjkKbxx/BY5ld9nc3avc57pocS6/JO5ITlgmb+cJ4OLRk8dLLnMBAQin8y9aTeoyWqk5ZYNTXDe3Rk7WcH6pTdMKYXf1q7k4tcAh8tIyXJ+kSZdi+xvtVNu1A/YQHYsPZ9Nm24k2mB5TwaUpIqw6QWVXJ3Uim95EsKAlvpIvsA2Ekef6kazbLk0bxaPojjMS7jOdMGMdD1WTEN++jeyRR9fjDmytVtxS9DnlvjqGseOitGAv3Md0dTVq+j+LDuHZu1DpUx7ZUlLSau+spObkt6jmnBlRzRVJ4MT+bGpJf5ReiXAFRoMp9pHmsiA8geMoGLJlxNp8werHihkAdnDGXmU6t4p3hPnffMTEnixnF94nE4vpOaHIxrwgJw+bl5PL38s5N+n7zMFHYdrjgFEZ06gleTlB6jpiU1OUhqsg8WdDQmgVjSEgfhiBKOKKGk9jl4K0iYEVLMzKSXmRhYzTFSeLhqOguqpnCATDZGenNODizc25cefQcztSCfq0d2q+ksmwo8dO1wALJiLGDYMzdxm2HMieZ+fTAje2Zz6cDOTe/ciJduGceO/WWnKKpTY/HssSzfuteadYw5RSxpaab3t+zlq72ySYoaZVJRFUaVRv9bWrz+cwq3H+DWCQNqht7+85Mread4D9t+No1IRKmoipzQ5v3Lv5Vw/ysf88lPp7b4C+/D0kP0z8uoE2u0yqoIEdWauHfsP0Y4ovTsVHc0xf6jlQ2ut3O0oooDZccJinBWSpCs1KY7Kw7IPM6/lT9S0x+lPCmLwJg7yLjgX7g5PZeba/b0qsdnNeNYZ47tXdPvZep5X+GlDV/UGdZqEl9SMMBVw7ud9PvkZqSQm5FYQ8qH5HdgSL5N5W/MqeKbpEVEJgMPAUHgCVX92an+jEhEWfj4T1m7fT+CEqKKALWTSLxM7Cm3pV5XwvqPAR58v3a7P9A/CPfe81K9veq+bmYQfjJnSZPvHbusYRGECIISqNmOuCl7Iq5M3btq1DspgqrUiVTdvg/8wzAIpSMpGQRSMgknZ6ApGUgog6TUTLYfKOey8qVMCq7m+fDFvBUu4Mc330ZqTqdGIm1a9RDi687vwe0TB9A7N51LBuQ18Spj/KOgWwcKdxxs9bBrY84k0pLhhvEiIkFgMzAB2AGsAq5T1U2x9h81apSuXr26xZ9TuqWQLr8bdzKhmgYc0VQypJzNkXwmVv4XG+dNOmHSN2PMibbsPoIq9MvLOO2fJSJrVHVU03saEx9++dU4HyhR1a0AIrIQuBKImbS0Vpe+Bbx+wQI27q4iPacLxbsrmDw0n7c376ZPbgZHKqr44mA5gYBw9cjuCLD8k71s+vwguRlpZKUmsWXPES4f1JkDZVWEkgKUfHmYqQVdKa8Ms/yTfXTPSaMyrBwpryIcUY5UhOl4VjIffXGYnp3SWfnJXgZ16UBSUKgKK4fLq8jrkEJ2WoiyyjCj+3Xig08P0C8vg9Wf7mdUz2z2HKlkx/4ycjNCLNuyl4mDO7NtbxmDu3YgPSVIl6xUikoPsaSwlAGdM1j0wQ52Hy4jgDKuXw5j+uRQWXWcNdv2cayiivEDc3mtqJTBXTPp1jGVsELnzBDHKqpIDgp/WL2djFASF/TOYcF7WwEIEuEsysmQctIpIz1qu/o+heNumn1jTHP1Pfv0JyvG+IVfalpmAJNV9Ub3+J+AC1R1dtQ+NwE3AfTo0WPkp59+GpdY25uyyjClB8vo08QX66K1O7n12XU8N+tCVm3bx/e/1q+NIjTGNJfVtJhE55ealiap6uPA4+A1D8U5nHYjLRRsMmEBmD48n+nD8wH4ahvNQmuMMebM4pcxtzuB6OlOu7kyY4wxxrQTfklaVgH9RaS3iISAa4HFcY7JGGOMMW3IF81DqlolIrOBV/GGPC9Q1aI4h2WMMcaYNuSLpAVAVV8C6k9sYowxxph2wi/NQ8YYY4xp5yxpMcYYY4wvWNJijDHGGF+wpMUYY4wxvuCLGXFbSkR2AyczJW4usOcUhdPWLPb4sNjbnl/jhsSNvaeqnh3vIIxpyBmZtJwsEVnt16msLfb4sNjbnl/jBn/Hbkw8WfOQMcYYY3zBkhZjjDHG+IIlLbE9Hu8AToLFHh8We9vza9zg79iNiRvr02KMMcYYX7CaFmOMMcb4giUtxhhjjPEFS1qiiMhkEflYREpE5K54xxOLiGwTkQ0isk5EVruyHBF5TUSK3X22KxcRedgdT6GIjGjjWBeIyC4R2RhV1uJYReQGt3+xiNwQx9jnishOd+7XicjUqOfudrF/LCKTosrb/JoSke4i8qaIbBKRIhG5xZUn/LlvJPaEP/cikioiK0VkvYt9nivvLSIrXBzPikjIlae4xyXu+V5NHZMx7Z6q2s3r1xMEtgB9gBCwHjg33nHFiHMbkFuv7H7gLrd9F/Cfbnsq8DIgwGhgRRvHejEwAtjY2liBHGCru89229lxin0ucHuMfc9110sK0NtdR8F4XVNAF2CE284ENrsYE/7cNxJ7wp97d/4y3HYysMKdzz8A17ry+cD33Pb3gflu+1rg2caO6XRfN3azmx9uVtNS63ygRFW3qmolsBC4Ms4xNdeVwFNu+ylgelT5b9WzHOgoIl3aKihVfRvYV6+4pbFOAl5T1X2quh94DZgcp9gbciWwUFUrVPUToATveorLNaWqpar6gds+DHwI5OODc99I7A1JmHPvzt8R9zDZ3RS4FHjeldc/79V/j+eBy0REGjkmY9o9S1pq5QPbox7voPEvy3hRYKmIrBGRm1xZZ1UtddtfAJ3ddiIeU0tjTbRjmO2aUBZUN6+QwLG7JofheP/1++rc14sdfHDuRSQoIuuAXXhJ3hbggKpWxYijJkb3/EGgU7xiN8YPLGnxn7GqOgKYAvxARC6OflJVFS+xSXh+itX5FdAXGAaUAj+PbziNE5EM4AXgVlU9FP1cop/7GLH74tyralhVhwHd8GpHzolzSMacUSxpqbUT6B71uJsrSyiqutPd7wL+hPfF+GV1s4+73+V2T8RjammsCXMMqvql+1GKAP9DbZV9wsUuIsl4P/q/V9U/umJfnPtYsfvp3AOo6gHgTeBCvOa2pBhx1MTonu8A7CWBrnljEo0lLbVWAf1dT/8QXse4xXGOqQ4RSReRzOptYCKwES/O6pEdNwB/dtuLgevd6JDRwMGo5oF4aWmsrwITRSTbNQlMdGVtrl5/oKvwzj14sV/rRoP0BvoDK4nTNeX6RfwG+FBVH4x6KuHPfUOx++Hci8jZItLRbacBE/D65LwJzHC71T/v1X+PGcBfXQ1YQ8dkjIl3T+BEuuGNotiM1w59T7zjiRFfH7xRBeuBouoY8drB3wCKgdeBHFcuwGPueDYAo9o43mfwqvKP47XLz2xNrMB38DojlgDfjmPsv3OxFeL9sHSJ2v8eF/vHwJR4XlPAWLymn0JgnbtN9cO5byT2hD/3QAGw1sW4EfiRK++Dl3SUAM8BKa481T0ucc/3aeqY7Ga39n6zafyNMcYY4wvWPGSMMcYYX7CkxRhjjDG+YEmLMcYYY3zBkhZjjDHG+IIlLcYYY4zxBUtajAFEJBy1gvC6plYFFpFZInL9KfjcbSKSe7Lv494rXURed9vvRk1oZowxZwT7UjPGU6be9OvNoqrzT2cwrXQh8L6bCO6o1q53Y4wxZwSraTGmEa4m5H4R2SAiK0WknyufKyK3u+2bRWSTW8xvoSvLEZFFrmy5iBS48k4islREikTkCbyJ3ao/61vuM9aJyK/d4ntBEflfEdnoYvjXGDH2dYv0PQ38I7AGGOreJ++0nyRjjGkjlrQY40mr1zx0TdRzB1X1POBR4BcxXnsXMFxVC4BZrmwesNaV/RD4rSv/MfCuqg7GWzuqB4CIDAKuAS5yNT5h4Jt4CwTmq+oQF8OT9T9cVbe416zBW5PnKWCmqg5Tb40qY4w5I1jzkDGexpqHnom6/+8YzxcCvxeRRcAiVzYWuBpAVf/qaliygIuBv3flS0Rkv9v/MmAksMpbfoc0vAUN/wL0EZFHgCXA0kaOIU9V97pand80dcDGGOM3VtNiTNO0ge1q0/DW7hmBl3S05p8BAZ5ytSPDVHWgqs5V1f3AUOBveLU4T5zwQpH5IrIRb4HAdcBk4MVYTUnGGONnlrQY07Rrou7fj35CRAJAd1V9E7gT6ABkAO/gNe8gIl8D9qjqIeBtvH4niMgUINu91RvAjOo+KK5PTE83siigqi8Ac/ASozpUdRZec9S9wHRgiUt8YtUKGWOMb1nzkDGeNFdLUe0VVa0e9pwtIoVABXBdvdcFgadFpANebcnDqnpAROYCC9zrjgE3uP3nAc+ISBGwDPgMQFU3icgcYKlLhI4DPwDKgCddGcDdDcR/CV6/mXHAWy0/fGOMSXy2yrMxjRCRbcAoVd0T71iMMaa9s+YhY4wxxviC1bQYY4wxxhespsUYY4wxvmBJizHGGGN8wZIWY4wxxviCJS3GGGOM8QVLWowxxhjjC/8PTzJQEol1dvUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "print('length of scores: ', len(scores), ', len of avg_scores: ', len(avg_scores))\n",
    "\n",
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "plt.plot(np.arange(1, len(scores)+1), scores, label=\"Score\")\n",
    "plt.plot(np.arange(1, len(avg_scores)+1), avg_scores, label=\"Avg on 100 episodes\")\n",
    "plt.legend(bbox_to_anchor=(1.05, 1)) \n",
    "plt.ylabel('Score')\n",
    "plt.xlabel('Episodes #')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 1\tAverage Score: 2493.66,\tScore: 2493.66 \tTime: 00:00:13\n",
      "Episode 2\tAverage Score: 2511.80,\tScore: 2529.95 \tTime: 00:00:13\n",
      "Episode 3\tAverage Score: 2517.11,\tScore: 2527.71 \tTime: 00:00:13\n",
      "Episode 4\tAverage Score: 1924.03,\tScore: 144.82 \tTime: 00:00:01\n",
      "Episode 5\tAverage Score: 2040.35,\tScore: 2505.62 \tTime: 00:00:13\n",
      "Episode 6\tAverage Score: 2124.66,\tScore: 2546.23 \tTime: 00:00:13\n",
      "Episode 7\tAverage Score: 2179.24,\tScore: 2506.69 \tTime: 00:00:13\n",
      "Episode 8\tAverage Score: 2219.78,\tScore: 2503.57 \tTime: 00:00:13\n",
      "Episode 9\tAverage Score: 2254.13,\tScore: 2528.94 \tTime: 00:00:13\n",
      "Episode 10\tAverage Score: 2282.75,\tScore: 2540.31 \tTime: 00:00:13\n"
     ]
    }
   ],
   "source": [
    "from collections import deque\n",
    "import os\n",
    "\n",
    "def play(env, agent, n_episodes):\n",
    "    state = env.reset()\n",
    "    \n",
    "    scores_deque = deque(maxlen=100)\n",
    "    scores = []\n",
    "\n",
    "    for i_episode in range(1, n_episodes+1):\n",
    "        state = env.reset()        \n",
    "        score = 0\n",
    "        \n",
    "        time_start = time.time()\n",
    "        \n",
    "        while True:\n",
    "            action = agent.select_action(np.array(state))\n",
    "            env.render()\n",
    "            time.sleep(0.01)\n",
    "            next_state, reward, done, _ = env.step(action)\n",
    "            state = next_state\n",
    "            score += reward\n",
    "            if done:\n",
    "                break \n",
    "\n",
    "        s = (int)(time.time() - time_start)\n",
    "        \n",
    "        scores_deque.append(score)\n",
    "        scores.append(score)\n",
    "        \n",
    "        print('Episode {}\\tAverage Score: {:.2f},\\tScore: {:.2f} \\tTime: {:02}:{:02}:{:02}'\\\n",
    "                  .format(i_episode, np.mean(scores_deque), score, s//3600, s%3600//60, s%60))  \n",
    "\n",
    "play(env=env, agent=agent, n_episodes=10)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
